# -*- coding: utf-8 -*-
"""
Created on 2021-10-13 00:50:17
---------
@summary: 分布式爬取公共管理学院
---------
@author: 大龙
"""
import random
import re
import time

from items.scunews_item import SCUNewsDataItem
import feapder
import sys
from constants import PAGE_XPATH_CONTENT,PAGE_XPATH_TITLE,PAGE_XPATH_SUBDATE,LIST_XPATH_DIC,LIST_XPATH_URL_DIC
from lxml import etree
sys.path.append('..')
# mystr="faasd 时间： 123"
# print(mystr[mystr.rfind('时间：')+3:])
class CsSpiderInterface(feapder.BatchSpider):
    def __init__(self,*args,**kwargs):
        super().__init__(*args,**kwargs)
        self.acadamyUrl=None
        self.type=None
        self.source=None


    def start_requests(self, task):
        id, url, type,source,acadamyUrl= task
        yield feapder.Request(url, task_id=id,type=type,source=source,acadamyUrl=acadamyUrl)

    def parse(self, request, response):
        print(response)
        items=response.json['items']
        for newsitem in items:
            item = SCUNewsDataItem()
            item['D_SOURCE']=request.source
            item['D_URL']=newsitem['url']
            item['D_TITLE']=newsitem['title']
            item['D_TYPE']=request.type
            item['D_SUBDATE']=newsitem['published_at']
            item['D_CONTENT']=re.sub(r'\\n|\\r|\\', "",newsitem['raw_html'])
            if(len(newsitem['photos'])!=0):
                item['D_PHOTO']=newsitem['photos'][0]
            yield item
        # 更新任务状态
        yield self.update_task_batch(request.task_id, 1)



    def get_date(self, mystr):
        """
        @summary:去除爬取日期前面的“发布日期：”
        --------
        @str:待去除字符串
        --------
        @result:str
        """
        if len(mystr[mystr.rfind('时间：')+3:])<len(mystr[mystr.rfind('：')+1:]) != 0 and mystr.rfind('时间：') != 0:
            return mystr[mystr.rfind('时间：')+3:mystr.rfind('时间：')+14]
        else:
            return mystr[mystr.rfind('：')+1:mystr.rfind('：')+12]

    def exception_request(self, request, response):
        """
        @summary: 请求或者parser里解析出异常的request
        ---------
        @param request:
        @param response:
        ---------
        @result: request / callback / None (返回值必须可迭代)
        """

        pass

    def failed_request(self, request, response):
        """
        @summary: 超过最大重试次数的request
        ---------
        @param request:
        ---------
        @result: request / item / callback / None (返回值必须可迭代)
        """

        yield request
        yield self.update_task_batch(request.task_id, -1)  # 更新任务状态为-1
